/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package templatemover;

import java.net.*;
import java.io.*;
import java.net.CookieManager;
import java.util.*;
import org.xml.sax.*;
import org.xml.sax.helpers.DefaultHandler;
import javax.xml.parsers.*;
import org.w3c.dom.*;
import org.apache.commons.lang.StringEscapeUtils;

/**
 *
 * @author jhuss
 */
public class tmComm {

    //read wait is only used when the server is lagged.
    public static final int READ_WAIT = 5000;
    public static final int READ_MAX_WAIT = 100000;
    public static final int WRITE_WAIT = 6000;
    public static final int LOGIN_WAIT = 4000;
    public static final int MAX_LAG = 5; //As suggested at http://www.mediawiki.org/wiki/Manual:Maxlag_parameter
    //Extra info for uploading imageFiles to Wikipedia
    public static final String pdbImageCopyright = "www.pdb.org {{PD-release}}";
    public static final String WIKI_FILE_UPLOAD = "http://en.wikipedia.org/wiki/Special:Upload";
    public static final String WIKI_NO_IMAGE = "No file by this name exists;";
    public static final String WIKI_NO_IMAGE2 = "Wikipedia does not have an Image page with this exact title.";
    private URL baseURL;
    private CookieStore cStore;
    private String userName;
    private String password;
    private Date nextComm;
    private Random myRand;

    /** Creates a new instance of tmComm */
    public tmComm(String uName, String pWord, String wUrl) {
        myRand = new Random();
        nextComm = new Date();
        userName = uName;
        password = pWord;
        try {
            baseURL = new URL(wUrl);
        } catch (Exception e) {

            System.out.println("Problem with the URL:" + wUrl);
            e.printStackTrace();
        }
        java.net.CookieManager cm = new java.net.CookieManager();
        cm.setCookiePolicy(CookiePolicy.ACCEPT_ALL);
        java.net.CookieHandler.setDefault(cm);
        cStore = cm.getCookieStore();

    }

    private void updateWait(int wait) {
        Date now = new Date();
        nextComm = new Date(now.getTime() + wait);

    }

    private void allowComm() {
        //this function makes the bot wait until it is allowed to perform the next communication with Wiki.
        Date now = new Date();

        long wait = nextComm.getTime() - now.getTime();
        //System.out.println("Wait = " + wait);
        double waitTime = ((double) wait) / 1000;
        if (wait > 0) {
            System.out.println("Waiting " + waitTime + " seconds for server cooldown.");
            System.out.flush();
            long left = wait;
            while (left > 10) {
                System.out.print(".");
                System.out.print(" ");
                System.out.flush();
                try {
                    if (left > 1000) {
                        Thread.currentThread().sleep(1000);
                        left = left - 1000;
                    } else {
                        Thread.currentThread().sleep(left);
                        left = 0;

                    }
                } catch (InterruptedException ex) {
                    System.out.println("Interrupted While Sleeping. Non-hazardous Error (ignored).");
                //ex.printStackTrace();
                }


            }
            Date time2 = new Date();
            long actWait = time2.getTime() - now.getTime();
            System.out.println(" (" + actWait + " milliseconds.)");

        }
        return;

    }    // something to handle cookies
    private HashMap cookies = new HashMap(12);

    private void setCookies(URLConnection u, Map map) {


        Iterator i = map.entrySet().iterator();
        StringBuilder cookie = new StringBuilder(100);
        while (i.hasNext()) {
            Map.Entry entry = (Map.Entry) i.next();
            cookie.append(entry.getKey());
            cookie.append("=");
            cookie.append(entry.getValue());
            cookie.append("; ");
        }
        u.setRequestProperty("Cookie", cookie.toString());

    // enable gzip compression
    //u.setRequestProperty("Accept-encoding", "gzip");
    }

    private void grabCookies(URLConnection u, Map map) {
        // reset the cookie store
        //Don't really want to reset here - we will just overwrite cookies instead.
        //map.clear();
        String headerName = null;
        for (int i = 1; (headerName = u.getHeaderFieldKey(i)) != null; i++) {
            if (headerName.equals("Set-Cookie")) {
                String cookie = u.getHeaderField(i);

                // _session cookie is for cookies2, otherwise this causes problems
                //if (cookie.indexOf("_session") != -1 && map == cookies)
                //    continue;

                cookie = cookie.substring(0, cookie.indexOf(";"));
                String name = cookie.substring(0, cookie.indexOf("="));
                String value = cookie.substring(cookie.indexOf("=") + 1, cookie.length());
                map.put(name, value);
            }
        }
    }

    /** Logs the user into wikipedia, this function is not needed to be called before upload page
     * as upload page will take care of the login by iteself
     */
    @SuppressWarnings("static-access")
    public void wikiLogin() {
        //Quick method to log the bot in.  Sets cookies as appropriate.
        URL loginURL;
        HttpURLConnection loginConn;
        String webLineSep = "\r\n";

        String data;
        this.allowComm();

        //So now we need to download the login page first and collect up the cookies.  otherwise Wikipedia
        //will think that we don't allow the setting of cookies.... Too bad. 

        try {
            loginURL = new URL(baseURL, "index.php?title=Special:Userlogin");
            loginConn = (HttpURLConnection) loginURL.openConnection();
            grabCookies(loginConn, cookies);
            loginConn.getContent(); //Should grab cookies for us.
            BufferedReader br = new BufferedReader(new InputStreamReader(loginConn.getInputStream()));
            String info = "";
            //check out the response for any errors
            while (br.ready()) {
                info = info + br.readLine() + "\n";
            }

        } catch (Exception e) {
            System.out.println("Failed Login.");
            e.printStackTrace();
        }


        try {
            loginURL = new URL(baseURL, "index.php?title=Special:Userlogin&action=submitlogin&type=login");
            loginConn = (HttpURLConnection) loginURL.openConnection();
            //loginConn = loginURL.openConnection();

            //loginConn.setDoOutput(true);
            loginConn.setRequestMethod("POST");
            setCookies(loginConn, cookies);

            String bNum = "";
            for (int i = 0; i < 12; i++) {
                bNum = bNum + myRand.nextInt(10);
            }

            String fieldBreak = "---------------------------" + bNum;
            //Now we need to build up all the info we are sending...
            String conDisp = "Content-Disposition: form-data; ";
            String sendData = "";
            sendData = sendData + "--" + fieldBreak + webLineSep;
            sendData = sendData + conDisp + "name=\"wpName\"" + webLineSep;
            sendData = sendData + webLineSep;
            sendData = sendData + userName + webLineSep;
            sendData = sendData + "--" + fieldBreak + webLineSep;

            sendData = sendData + conDisp + "name=\"wpPassword\"" + webLineSep;
            sendData = sendData + webLineSep;
            sendData = sendData + password + webLineSep;
            sendData = sendData + "--" + fieldBreak + webLineSep;

            sendData = sendData + conDisp + "name=\"wpRemember\"" + webLineSep;
            sendData = sendData + webLineSep;
            sendData = sendData + "1" + webLineSep;
            sendData = sendData + "--" + fieldBreak + webLineSep;

            //Last entry
            sendData = sendData + conDisp + "name=\"wpLoginattempt\"" + webLineSep;
            sendData = sendData + webLineSep;
            sendData = sendData + "Log in" + webLineSep;
            sendData = sendData + webLineSep + "--" + fieldBreak + "--" + webLineSep;

            //setup the communicator for output...
            loginConn.setDoOutput(true);
            loginConn.setDoInput(true);
            loginConn.setUseCaches(false);
            loginConn.setRequestProperty("Content-Length", String.valueOf(sendData.length()));
            loginConn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + fieldBreak);

            System.out.println("Posting Data...");


            OutputStream out = loginConn.getOutputStream();

            //PrintWriter outp = new PrintWriter(out);

            out.write(sendData.getBytes("UTF-8"));
            out.flush();
            out.close();





//            BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(loginConn.getOutputStream()));
//            data = "";
//            data = encode("wpName") + "=" + encode(userName);
//            data += "&" + encode("wpPassword") + "=" + encode(password);
//            data += "&" + encode("wpRemember") + "=" + encode("1");
//            data += "&" + encode("wpLoginattempt") + "=" + encode("Log in");
//            
//            bw.write(data);
//            bw.flush();
//            bw.close();
            grabCookies(loginConn, cookies);
            loginConn.getContent(); //Should grab cookies for us.
            BufferedReader br = new BufferedReader(new InputStreamReader(loginConn.getInputStream()));
            String info = "";
            //check out the response for any errors
            while (br.ready()) {
                info = info + br.readLine() + "\n";
            }


            this.updateWait(this.LOGIN_WAIT);

        } catch (Exception e) {
            System.out.println("Failed Login.");
            e.printStackTrace();
        }

    }

    /** Userful method to grab cookie data from the existing cookies and put it in a nice string
     * formatted for setrequest mothods.  Also logs the user in if the cookies have expired
     * or are missing.
     */
    private String checkCookies() {
        //This will get the necessary cookie data possibly logging back in.

        boolean reLogin = false;
        if (cookies.size() < 4) {
            reLogin = true;
        }
        if (reLogin) {
            this.wikiLogin();
        }

        return "";

    }

    private String getCookieData2() {
        //This will get the necessary cookie data possibly logging back in.
        String data = "";
        List<HttpCookie> cookies;
        cookies = cStore.getCookies();

        //First make sure none have expired...
        boolean reLogin = false;
        if (cookies.size() < 4) {
            reLogin = true;
        }
        for (int i = 0; i < cookies.size(); i++) {
            HttpCookie cookie = cookies.get(i);
            if (cookie.hasExpired() == true) {
                reLogin = true;
            }
        }

        if (reLogin) {
            this.wikiLogin();
        }


        for (int i = 0; i < cookies.size(); i++) {
            HttpCookie cookie = cookies.get(i);

            if (i == 0) {
                data = encode(cookie.getName().trim()) + "=" + encode(cookie.getValue());
            } else {
                data = data + ", " + encode(cookie.getName().trim()) + "=" + encode(cookie.getValue());
            }
        }
        //Begin Debug
        //System.out.println("Cookie Data: " + data); //DEBUG
        //End Debug
        return data;
    }

    public String getFullURL(String title) {
        URL editURL;
        String page = "";
        try {
            editURL = new URL(baseURL, "index.php?title=" + title);
            page = editURL.toString();
        } catch (MalformedURLException ex) {
            ex.printStackTrace();
        }
        return page;
    }

    public void uploadPage(String page, String text, String editComment) throws Exception {
        //wraps for error catching.
        boolean clear = false;
        int tries = 0;
        while (!clear) {
            try {
                if (tmMain.debugUpload == true) {
                    this.uploadPageH(page, text, editComment);
                } else {
                    System.out.println("Upload skipped due to Debug Upload being false.");
                }

                return;
            } catch (Exception e) {
                System.out.println("Encountered Error while Uploading:");
                System.out.println(e.getMessage());
                if (tries < 3) {
                    System.out.println("Trying " + (3 - tries) + " more time(s) in 10 seconds...");
                    this.updateWait(10000);
                    this.allowComm();
                    tries++;


                } else {
                    System.out.println("3 tries failed; Requiring User Input:");
                    tries = 0;

                    try {
                        BufferedReader inputLine = new BufferedReader(new InputStreamReader(System.in));
                        System.out.println("Try Again? (y/n)");
                        System.out.flush();
                        String input = inputLine.readLine();
                        //input = "y";
                        if (input.compareToIgnoreCase("y") == 0) {
                            clear = false;
                        } else {
                            clear = true;
                            throw new Exception("User Aborted Upload due to error.", e);

                        }

                    } catch (IOException ex) {
                        throw new Exception("Unrecoverable IO Error.", ex);

                    }
                }
            }
        }
        throw new Exception("Upload Aborted.");

    }

    /** Method to upload a page to the wiki server.  Should log the user in first if
     * the user is not already logged in.
     * text is the page text,
     * page is the Title of the page to be editted.
     * editComment is the summary comment for the page edit.
     */
    public void uploadPageH(String page, String text, String editComment) throws Exception {
        //Upload a page to wikipedia.
        URL editURL;
        URLConnection editConn;
        URL postURL;
        HttpURLConnection postConn;
        String lineSep = System.getProperty("line.separator");
        String webLineSep = "\r\n";

        this.allowComm();

        //First do a little url encoding...

        page = encode(page);
        //editComment = encode(editComment);
        //Make sure the cookies are up to date

        String info;
        BufferedReader br;
        this.checkCookies();

        //Now open up the edit page so we can get data from it...
        editURL = new URL(baseURL, "index.php?title=" + page + "&action=edit");
        editConn = editURL.openConnection();
        setCookies(editConn, cookies);

        //editConn.setRequestProperty("Cookie", this.getCookieData());
        editConn.getContent();

        //get our hidden form values...
//        BufferedReader br = new BufferedReader(new InputStreamReader(editConn.getInputStream()));
//
//        String info = "";
//        //check out the response for any errors
//        while (br.ready()) {
//            info = info + br.readLine() + lineSep;
//        }
//        
//        br.close();
        //Hashtable<String, String> hiddenVals = htmlParse2(info);
        Hashtable<String, String> hiddenVals = htmlParse(editConn.getInputStream());

        postURL = new URL(baseURL, "index.php?title=" + page + "&action=submit");
        postConn = (HttpURLConnection) postURL.openConnection();
        postConn.setRequestMethod("POST");
        setCookies(postConn, cookies);

        //postConn.setRequestProperty("Cookie", this.getCookieData());


        String bNum = "";
        for (int i = 0; i < 12; i++) {
            bNum = bNum + myRand.nextInt(10);
        }

        String fieldBreak = "---------------------------" + bNum;
        //Now we need to build up all the info we are sending...
        String conDisp = "Content-Disposition: form-data; ";
        String sendData = "";
        sendData = sendData + "--" + fieldBreak + webLineSep;
        sendData = sendData + conDisp + "name=\"wpTextbox1\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + text + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        sendData = sendData + conDisp + "name=\"wpSummary\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + editComment + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        sendData = sendData + conDisp + "name=\"wpSave\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + "1" + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        sendData = sendData + conDisp + "name=\"wpStarttime\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + hiddenVals.get("wpStarttime") + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        sendData = sendData + conDisp + "name=\"wpEdittime\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + hiddenVals.get("wpEdittime") + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        sendData = sendData + conDisp + "name=\"wpEditToken\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + hiddenVals.get("wpEditToken") + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        sendData = sendData + conDisp + "name=\"wpAutoSummary\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + hiddenVals.get("wpAutoSummary") + webLineSep;
        sendData = sendData + "--" + fieldBreak + webLineSep;

        //Last entry
        sendData = sendData + conDisp + "name=\"wpWatchthis\"" + webLineSep;
        sendData = sendData + webLineSep;
        sendData = sendData + "0" + webLineSep;
        sendData = sendData + webLineSep + "--" + fieldBreak + "--" + webLineSep;

        //setup the communicator for output...
        postConn.setDoOutput(true);
        postConn.setDoInput(true);
        postConn.setUseCaches(false);
        postConn.setRequestProperty("Content-Length", String.valueOf(sendData.length()));
        postConn.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + fieldBreak);

        System.out.println("Posting Data...");


        OutputStream out = postConn.getOutputStream();

        //PrintWriter outp = new PrintWriter(out);

        out.write(sendData.getBytes("UTF-8"));
        out.flush();
        out.close();

        System.out.println("Finished posting data.");

        //Now we need to read back the page..

        br = new BufferedReader(new InputStreamReader(postConn.getInputStream()));

        info = "";
        //check out the response for any errors
        while (br.ready()) {
            info = info + br.readLine() + lineSep;
        }

        if (info.indexOf("Sorry! We could not process your edit due to a loss of session data.") != -1) {

            System.out.println("Update failed. Loss of session data.");
            throw new Exception("Loss of Session data.");
        }

        postConn.getContent();
        grabCookies(postConn, cookies);

        this.updateWait(this.WRITE_WAIT);
        //Looks good.
        return;

    }

    private Hashtable<String, String> htmlParse2(String page) {
        Hashtable<String, String> data = new Hashtable<String, String>();
        //Ok, so we have to search through this document page for the desired data we want... 
        //Good times here... Bleh.

        int index1 = -2;

        while (index1 != -1) {
            index1 = page.indexOf("<input", index1 + 1);
            if (index1 == -1) {
                break;
            }
            int index2 = page.indexOf("/>",index1);
            if (index1 > index2) {
                //Got a problem here.
                break;
            }
            
            String entry = page.substring(index1,index2+2);
            index1 = index2+2; //move index 1 to the end of the entry.
            
            if (entry.indexOf("type=\"hidden\"") == -1 && entry.indexOf("type='hidden'") == -1 ) {
                //not the right kind of input.
                continue;
            }
            //Ok, looks Like we have an entry.. good times.. 
            String[] tokens = entry.split("\\s+");
            String val = "";
            String name = "";
                    
            for (int i =0; i < tokens.length; i++){
                if ((index2 =tokens[i].indexOf("value=\"")) != -1){
                    val = tokens[i].substring(index2+7,tokens[i].length()-1);
                    val = val.trim();
                }
                if ((index2 =tokens[i].indexOf("name=\"")) != -1){
                    name = tokens[i].substring(index2+6,tokens[i].length()-1);
                    name=name.trim();
                }
            }
            if (val.compareTo("") != 0 && name.compareTo("") != 0){
                data.put(name, val);
            }
            
            
        }

        return data;
    }

    private Hashtable<String, String> htmlParse(InputStream inPage) {
        Hashtable<String, String> data = null;
        System.out.println("Starting Web parse.");
        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
        Document importPage;
        try {
            DocumentBuilder builder = factory.newDocumentBuilder();
            importPage = builder.parse(inPage);
            //System.out.println("Retrieving Data...");
            data = this.htmlDataMine(importPage);


        } catch (Exception e) {
            e.printStackTrace();
        }

        return data;

    }

    private Hashtable<String, String> htmlDataMine(Document doc) throws Exception {
        Hashtable<String, String> vals = null;

        Element root = doc.getDocumentElement();
        NodeList inputList = root.getElementsByTagName("input");

        vals = new Hashtable<String, String>();

        for (int i = 0; i < inputList.getLength(); i++) {
            Element tmpElement = (Element) inputList.item(i);

            //System.out.println("Input #" + i +": " + tmpElement.getAttribute("name") + " = " + tmpElement.getAttribute("value"));
            if ("hidden".compareToIgnoreCase(tmpElement.getAttribute("type")) == 0) {
                if ("".compareToIgnoreCase(tmpElement.getAttribute("value").trim()) != 0) {
                    vals.put(tmpElement.getAttribute("name"), tmpElement.getAttribute("value"));
                }
            }

        }

        return vals;
    }

    public String downloadPage(String page) throws Exception {
        //wraps for error catching.
        boolean clear = false;
        int tries = 0;
        while (!clear) {

            try {
                String data = downloadPageH(page);
                return data;
            } catch (Exception e) {
                System.out.println("Encountered Error while Downloading:");
                System.out.println(e.getMessage());
                if (tries < 3) {
                    System.out.println("Trying " + (3 - tries) + " more time(s) in 10 seconds...");
                    this.updateWait(10000);
                    this.allowComm();
                    tries++;


                } else {
                    System.out.println("3 tries failed; Requiring User Input:");
                    tries = 0;

                    try {
                        BufferedReader inputLine = new BufferedReader(new InputStreamReader(System.in));
                        System.out.println("Try Again? (y/n)");
                        System.out.flush();
                        String input = inputLine.readLine();
                        //input = "y";
                        if (input.compareToIgnoreCase("y") == 0) {
                            clear = false;
                        } else {
                            clear = true;
                            throw new Exception("User Aborted Download due to error.", e);

                        }

                    } catch (IOException ex) {
                        throw new Exception("Unrecoverable IO Error.", ex);
                    //System.out.println("Unrecoverable IO Error.");
                    //ex.printStackTrace();
                    //System.exit(1);
                    }
                }
            }

        }
        throw new Exception("Unknown Download Error.");
    //return null;
    }

    /** Method to retrieve the code content of a wiki page.  returns null if the page
     * has no textbox (ie we are not allowed to create that page
     * returns and empty string if no code content existed for the page, or the page
     * did not yet exist, but we were allowed to create it.
     */
    private String downloadPageH(String page) throws Exception {
        //Returns the data of the indicated page. It is found in the wpTextbox1 area.
        String lineSep = System.getProperty("line.separator");
        page = encode(page);
        URL editURL;
        HttpURLConnection editConn;
        String data = "";

        //So max lag check is to speed up reading of the wiki pages, unless they are bogged down.
        //we will get a response of a single line if the server lags - we then need to wait for some amount
        //of time and repoll the server.
        editURL = new URL(baseURL, "index.php?title=" + page + "&action=edit&maxlag=" + this.MAX_LAG);

        MAX_LAG_CHECK:
        {
            boolean maxLagError = true;
            while (maxLagError) { //so go at least once.
                String fullData = "";
                this.allowComm();
                this.checkCookies();
                editConn = (HttpURLConnection) editURL.openConnection();
                setCookies(editConn, cookies);
                //editConn.setRequestProperty("Cookie", this.getCookieData());
                try {

                    editConn.getContent();
                    grabCookies(editConn, cookies);
                } catch (IOException ex) {

                    if (editConn.getResponseCode() == HttpURLConnection.HTTP_UNAVAILABLE) {
                        BufferedReader br;

                        br = new BufferedReader(new InputStreamReader(editConn.getErrorStream()));

                        String line;

                        int readCount = 0;
                        fullData = "";
                        while ((line = br.readLine()) != null) {
                            fullData = fullData + line + lineSep;
                            readCount++;
                        }
                        if (readCount < 3) { //only read one or two lines - something was wrong.
                            if (fullData.matches("Waiting for .*: .* seconds lagged\\s*")) {
                                //looks like we were lagged
                                System.out.println("Server Lagged on Read. " + fullData);
                                String[] words = fullData.split("\\s+");
                                int waitTime = 0;
                                if (words.length > 3) {
                                    try {
                                        waitTime = Integer.parseInt(words[3]) * 1000;
                                    } catch (NumberFormatException nfe) {
                                        waitTime = 0;
                                    }
                                }

                                if (waitTime < this.READ_WAIT) {
                                    waitTime = this.READ_WAIT;
                                }
                                if (waitTime > this.READ_MAX_WAIT) {
                                    waitTime = this.READ_MAX_WAIT;
                                }

                                this.updateWait(waitTime);
                                continue;
                            } else {
                                throw ex;
                            }
                        } else {
                            throw ex;
                        }
                    } else {
                        throw ex;
                    }
                }

                //if we get here, no lag occured.
                InputStream is = editConn.getInputStream();
                int numRead;
                byte[] input = new byte[1024];
                //String fullData = "";
                //ArrayList fullData = new ArrayList();
                byte[] byteData = new byte[0];
                byte[] tmp = null;
                while ((numRead = is.read(input)) != -1) {
                    tmp = byteData;
                    byteData = new byte[tmp.length + numRead];
                    System.arraycopy(tmp, 0, byteData, 0, tmp.length);
                    System.arraycopy(input, 0, byteData, tmp.length, numRead);
                }

//                BufferedReader br = new BufferedReader(new InputStreamReader(is));
//                String line;
//                fullData = "";
//
//                while ( (line = br.readLine()) != null) {
//                    fullData = fullData + line + lineSep;
//                }
//
                fullData = new String(byteData, "UTF-8");
                //System.err.println(fullData);
                //byte[] theData;
                //theData = fullData.getBytes("UTF-16");

                data = this.parseTextArea(fullData);
                //Now clean up this data to remove HTML escape sequences.
                data = StringEscapeUtils.unescapeHtml(data);
                maxLagError = false;
            }


        } //END MAX_LAG_CHECK

        if (data == null) {
            data = "";
        }
        return data;


    }

    private String parseTextArea(String fullData) {
        //Helper function to extract the page data from the textarea entry.
        String data = "";
        int index1 = fullData.indexOf("name=\"wpTextbox1\"");
        if (index1 == -1) {
            //have no match, likely the page does not exist and we are not allowed to create it.
            data = null;
            return data;
        }
        int index2 = fullData.indexOf(">", index1);
        index1 = fullData.indexOf("</textarea>", index2);
        data = fullData.substring(index2 + 1, index1);
        if (data.trim().compareTo("") == 0) {
            //blank page or empty textbox.
            data = "";

        }
        return data;
    }

    public static String encode(String in) {
        //Quick shortcut
        try {
            return URLEncoder.encode(in, "UTF-8");
        } catch (Exception e) {
            System.out.println("Error While Encoding.");
            return in;
        }
    }
}
